package com.thinkbig.crawler;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.util.ArrayList;
import java.util.Random;

public class FetchPages {

    /**
     * @param args
     * @throws InterruptedException
     */
    public static void main(String[] args) throws InterruptedException {
        ArrayList<String> urls = new ArrayList<String>();
        for (int i = 10000; i < 10002; i++) {
            urls.add(String.format("http://baike.baidu.com/view/%s.htm", i));
        }

        int threadNum = 1;
        while (threadNum-- > 0) {
            SubInst sub = new SubInst(urls);
            sub.start();
        }
    }
}

class SubInst extends Thread {
    private ArrayList<String> urlList = new ArrayList<String>();
    long time = new Random(NORM_PRIORITY).nextInt(500);

    public SubInst(ArrayList<String> urlList) {
        this.urlList = urlList;
    }

    public void run() {
        String url = "";
        while (true) {
            synchronized (urlList) {
                if (urlList.size() == 0) {
                    break;
                } else {
                    url = urlList.remove(0);
                }
            }
            if (url == null || url.length() == 0) {
                return;
            }

            try {
                fileGetContents(url);
                System.out.println(this.getName() + ":" + url);
                Thread.sleep(time);
                System.out.printf("sleep:%dms\n", time);
            } catch (IOException e) {
                e.printStackTrace();
                continue;
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }

    }

    public static void fileGetContents(String strUrl) throws IOException {
        URL url1 = null;
        BufferedReader reader = null;
        HttpURLConnection connection = null;
        InputStreamReader ins = null;

        String baseDir = "d:/webpages/";
        String[] segs = strUrl.split("/");
        String fileName = segs[segs.length - 1];
        String filePath = baseDir + fileName;

        try {
            FileWriter fw = new FileWriter(filePath);
            BufferedWriter bw = new BufferedWriter(fw);

            url1 = new URL(strUrl);
            connection = (HttpURLConnection) url1.openConnection();
            connection.setConnectTimeout(2 * 1000);
            connection.setReadTimeout(2 * 1000);
            connection.connect();
            ins = new InputStreamReader(connection.getInputStream(), "gbk");
            reader = new BufferedReader(ins);
            StringBuffer sb = new StringBuffer();
            String line;

            while ((line = reader.readLine()) != null) {
                sb.append(line + "\n");
            }
            bw.write(sb.toString());
            bw.close();
            fw.close();
        } catch (IOException e) {
            System.out.println("Error getURL:" + strUrl);
        } finally {
            if (ins != null) {
                try {
                    ins.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
            if (reader != null) {
                try {
                    reader.close();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
            if (connection != null) {
                connection.disconnect();
            }
        }
    }

}
